#!/usr/bin/env python
#coding=utf-8

import urllib, urllib2
import cookielib
import re, time
import threading
import gzip
from StringIO import StringIO
import hashlib
import sys

def get_content(res):
    ce = res.headers.get("Content-Encoding")
    if ce and 'zip' in ce:
        return gzip.GzipFile(fileobj=StringIO(res.read())).read() 
    return res.read()

class Venue():
    headers = { 'User-Agent' : """Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US; rv:1.9.2.3) Gecko/20100401 Firefox/3.6.3 GTB6"""}
    headers['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
    headers['Accept-Encoding'] = 'gzip,deflate'
    headers['Accept-Language'] = "zh,en-us;q=0.7,en;q=0.3"
    headers['Accept-Charset'] = "ISO-8859-1,utf-8;q=0.7,*;q=0.7"
    headers['Connection'] = "keep-alive"
    headers['Keep-Alive'] = "115"
    headers['Cache-Control'] = "no-cache"
    v_cat = re.compile(r"""searchKeyword\('(.*?)'\).*?href="(.*?)".*?""", re.S)
    v_cat_list = re.compile(r"""<a\s+href="(/detail/\d+-\d+)".*?>(.*?)</a>.*?<div\s+id="ppctel\d+".*?>(.*?)</div>(.*?)<br.*?name="entityXY"\s+value='(.*?)'.*?""", re.S)
    v_cat_list_page = re.compile(r"""<div class="pages">.*<a\s+href=(.*?fm=&p=)\d+>(\d+)</a>.*?</div>""", re.S)


    def _opener(self):
        """
        构建自动处理httpcookie的opener
        """
        cj = cookielib.CookieJar()
        opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj), urllib2.HTTPHandler())
        return opener

    def _wrap(self, url, param=None):
        return urllib2.Request(url, param, self.headers)

    def __init__(self, base , city):
        self.city = city
        self.base = base
        self.dups = set()
        c = city
        if "win" in sys.platform:
            c = city.decode("utf8")
        self.file = open(c+".txt", "wb")
        self.file.write("%s\n"%city)

    def fetch(self):
        op = self._opener()
        try:
            r = op.open(self._wrap(self.base + urllib.quote(self.city)))
            content = get_content(r)
            r.close()
    
            cats = self.v_cat.findall(content)
            print "found", len(cats), "categories"
            for m in cats:
                self.fetchList(op, m[0], "http://www.aibang.com" + m[1])
        except:
            print "error occured"
        finally:
            self.file.close()

        print "finished"

    def fetchList(self, op, cat, url):
        #处理分页数据
        urls = [url]
        max_page = 0
        while 1:
            time.sleep(5)
            try:
                if not urls: break
    
                u = urls.pop()
                print "fetching ", u
                r = op.open(self._wrap(u))
                content = get_content(r)
                r.close()
    
                pois = self.v_cat_list.findall(content)
                pages = self.v_cat_list_page.findall(content)

                if not pois:
                    print "error find poi:", u
                    time.sleep(5)

                print "fetched ", u
                for m in pois:
                    print >> self.file, cat, "\t", "\t".join(m)
                    print cat, "\t", "\t".join(m)

                self.file.flush()
    
                if len(pages) > 0:
                    url_prefix, max_p =pages[0]
                    max_p = int(max_p)
                    if max_p > max_page:
                        max_page = max_p
                        for x in range(max_page, 0, -1):
                            _url = "http://www.aibang.com%s%s" % (url_prefix, x)
                            _key = hashlib.md5(_url).hexdigest()
                            if _key in self.dups:
                                break
                            self.dups.add(_key)
                            urls.append(_url)
            except BaseException, msg:
                print "error", msg
                time.sleep(5)

def fetch(city):
    v = Venue("http://www.aibang.com/?area=common&cmd=services&city=", city)
    v.fetch()

if __name__ == "__main__":
#    districts = [
#        ['北京'],
#        ['重庆'],
#        ['上海'],
#        ['天津'],
#        ['安徽','合肥','安庆','蚌埠','亳州','巢湖','池州','滁州','阜阳','淮北','淮南','黄山','六安','马鞍山','宿州','铜陵','芜湖','宣城'],
#        ['福建','福州','龙岩','南平','宁德','莆田','泉州','三明','厦门','漳州'],
#        ['甘肃','白银','兰州','酒泉','金昌','武威'],
#        ['广东','广州','潮州','东莞','佛山','河源','惠州','江门','揭阳','茂名','梅州','清远','汕头','汕尾','韶关','深圳','阳江','云浮','湛江','肇庆','中山','珠海'],
#        ['广西','南宁','北海','防城港','桂林','河池','柳州'],
#        ['贵州','贵阳'],['海南','海口','三亚'],
#        ['河北','石家庄','保定','沧州','承德','邯郸','衡水','廊坊','秦皇岛','唐山','邢台','张家口'],
#        ['河南','郑州','安阳','鹤壁','济源','焦作','开封','洛阳','漯河','南阳','平顶山','濮阳','三门峡','商丘','新乡','信阳','许昌','周口','驻马店'],
#        ['黑龙江','哈尔滨','大庆','鹤岗','黑河','鸡西','佳木斯','牡丹江','七台河','齐齐哈尔','双鸭山','绥化','伊春'],
#        ['湖北','武汉','鄂州','恩施','黄冈','黄石','荆门','荆州','潜江','十堰','随州','神农架','天门','仙桃','咸宁','襄樊','孝感','宜昌'],
#        ['湖南','长沙','常德','郴州','衡阳','怀化','娄底','邵阳','湘潭','益阳','永州','岳阳','张家界','株洲'],
#        ['吉林','长春','白城','白山','吉林','辽源','四平','松原','通化','延吉'],
#        ['江苏','南京','常州','淮安','连云港','南通','苏州','宿迁','泰州','无锡','徐州','盐城','扬州','镇江'],
#        ['江西','南昌','抚州','赣州','吉安','景德镇','九江','萍乡','上饶','新余','宜春','鹰潭'],
#        ['辽宁','沈阳','鞍山','本溪','朝阳','大连','丹东','抚顺','阜新','葫芦岛','锦州','辽阳','盘锦','铁岭','营口'],
#        ['内蒙古','呼和浩特','包头','赤峰'],
#        ['宁夏','银川'],
#        ['青海','西宁'],
#        ['山东','济南','滨州','德州','东营','菏泽','济宁','莱芜','聊城','临沂','青岛','日照','泰安','威海','潍坊','烟台','枣庄','淄博'],
#        ['山西','太原','长治','大同','晋城','晋中','临汾','吕梁','朔州','忻州','阳泉','运城'],
#        ['陕西','西安','安康','宝鸡','汉中','商洛','铜川','渭南','咸阳','延安','榆林'],
#        ['四川','成都','巴中','达州','德阳','广安','广元','乐山','泸州','眉山','绵阳','内江','南充','攀枝花','遂宁','雅安','宜宾','资阳','自贡'],
#        ['西藏','拉萨','日喀则'],
#        ['新疆','乌鲁木齐','吐鲁番'],
#        ['云南','昆明','大理','曲靖','西双版纳','玉溪','昭通'],['浙江','杭州','湖州','嘉兴','金华','丽水','宁波','衢州','绍兴','台州','温州','舟山'],
#    ]
#
#    for province in districts:
#        if len(province) > 1:
#            for city in province[1:]:
#                fetch(city)
#        elif len(province) == 1:
#                fetch(province[0])

    fetch("天津")  #修改城市地址为自己的分到的城市地址